#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Datetime: 2021/10/05 14:17
# @Author  : CHEN Wang
# @Site    :
# @File    : crypto_timing_factor_strategies_select.py
# @Software: PyCharm

"""
脚本说明： 数字货币择时指标精选指标筛选及对应信号转换方法解析
"""

import os
import pandas as pd
from pylab import mpl
import seaborn as sns
from matplotlib import pyplot as plt
from quant_researcher.quant.project_tool.time_tool import get_yesterday
from quant_researcher.quant.project_tool.localize import TEST_DIR, DATA_DIR, TA_TEST_DIR
from quant_researcher.quant.datasource_fetch.crypto_api.glassnode import get_prices, get_ret, all_http, get_indicators
from quant_researcher.quant.datasource_fetch.crypto_api import sanbase
from quant_researcher.quant.factors.factor_analysis.factor_analyser.crypto_timing_factor_batch_test import get_already_tested_indic, get_already_tested_ta_indic
from quant_researcher.quant.datasource_fetch.crypto_api.self_defined import onchain_metrics_list, trading_metrics_list, social_metrics_list
from quant_researcher.quant.factors.factor_analysis.factor_correlation.fac_corr_analysis import factor_clustering

mpl.rcParams['font.sans-serif'] = ['SimHei']  # 设置字体为黑体
mpl.rcParams['axes.unicode_minus'] = False  # 解决中文字体负号显示不正常问题


def delete_each_indic_selected_strategies():
    """

    :return:
    """

    already_tested_indic = get_already_tested_indic()
    for factor_name in already_tested_indic:
        for factor_type in ['origin', 'diff_d', 'diff_w', 'diff_m']:
            factor_name1 = factor_name + f'_{factor_type}'
            file_path = os.path.join(TEST_DIR, f'{factor_name1}')
            file_name = os.path.join(file_path, f'backtest_result_selected_best_strategy.xlsx')
            if os.path.exists(file_name):  # 如果文件存在
                os.remove(file_name)
            file_name = os.path.join(file_path, f'backtest_result_selected_buy_strategy.xlsx')
            if os.path.exists(file_name):  # 如果文件存在
                os.remove(file_name)
            file_name = os.path.join(file_path, f'backtest_result_selected_sell_strategy.xlsx')
            if os.path.exists(file_name):  # 如果文件存在
                os.remove(file_name)

        file_path = os.path.join(TEST_DIR, f'{factor_name}')
        file_name = os.path.join(file_path, f'all_selected_best_strategy.xlsx')
        if os.path.exists(file_name):  # 如果文件存在
            os.remove(file_name)
        file_name = os.path.join(file_path, f'all_selected_buy_strategy.xlsx')
        if os.path.exists(file_name):  # 如果文件存在
            os.remove(file_name)
        file_name = os.path.join(file_path, f'all_selected_sell_strategy.xlsx')
        if os.path.exists(file_name):  # 如果文件存在
            os.remove(file_name)

    return


def get_each_indic_selected_strategies(indic='all', if_print=False):
    """
    获取每个指标对应精选策略, 买点策略，卖点策略

    :param if_print:
    :return:
    """

    already_tested_normal_indic = get_already_tested_indic()
    already_tested_ta_indic = get_already_tested_ta_indic()
    if indic == 'all':
        already_tested_indic = already_tested_normal_indic + already_tested_ta_indic
    elif indic == 'ta':
        already_tested_indic = already_tested_ta_indic
    elif indic == 'normal':
        already_tested_indic = already_tested_normal_indic

    for factor_name in already_tested_indic:
        selected_best_strategy_list = []
        selected_buy_strategy_list = []
        selected_sell_strategy_list = []
        if factor_name in already_tested_normal_indic:
            factor_type_list = ['origin', 'diff_d', 'diff_w', 'diff_m']
        else:
            file_path_temp = os.path.join(TA_TEST_DIR, f'{factor_name.__name__}')
            factor_type_list = [x for x in os.listdir(file_path_temp) if ('xlsx' not in x) and ('csv' not in x) and ('.DS_Store' not in x)]

        for factor_type in factor_type_list:
            if factor_name in already_tested_normal_indic:
                factor_name1 = factor_name + f'_{factor_type}'
                file_path = os.path.join(TEST_DIR, f'{factor_name1}')
            else:
                file_path = os.path.join(TA_TEST_DIR, f'{factor_name.__name__}\\{factor_type}')
            file_name = os.path.join(file_path, f'backtest_result_all')
            if not os.path.exists(f'{file_name}.xlsx'):
                continue
            all_perf = pd.read_excel(f'{file_name}.xlsx', index_col='Unnamed: 0')
            all_perf = all_perf.astype(str)
            commission = all_perf.index[['费后年化收益率' in i for i in all_perf.index]][0].split('%')[0]
            all_perf = all_perf.T
            # 部分值从字符串转换成数值
            all_perf[f'{commission}%费后年化收益率'] = [float(i.replace('%', '')) for i in all_perf[f'{commission}%费后年化收益率']]
            all_perf[f'{commission}%费后最大回撤'] = [float(i.replace('%', '')) for i in all_perf[f'{commission}%费后最大回撤']]
            all_perf[f'主动管理超额年化收益率'] = [float(i.replace('%', '')) for i in all_perf[f'主动管理超额年化收益率']]
            all_perf[f'交易次数'] = [int(i) if i != 'nan' else 0 for i in all_perf[f'交易次数']]
            all_perf[f'平均多少天交易一次'] = [int(i) if i != 'nan' else 0 for i in all_perf[f'平均多少天交易一次']]
            all_perf[f'回测总天数'] = all_perf[f'交易次数'] * all_perf[f'平均多少天交易一次']
            all_perf[f'总胜率'] = [float(i.replace('%', '')) for i in all_perf[f'总胜率']]
            all_perf[f'买入信号胜率'] = [float(i.replace('%', '')) for i in all_perf[f'买入信号胜率']]
            all_perf[f'买入最大亏损'] = [float(i.replace('%', '')) for i in all_perf[f'买入最大亏损']]
            all_perf[f'卖出信号胜率'] = [float(i.replace('%', '')) for i in all_perf[f'卖出信号胜率']]
            all_perf[f'卖出最大亏损'] = [float(i.replace('%', '')) for i in all_perf[f'卖出最大亏损']]
            all_perf[f'最佳卖出占所有最佳卖点比例'] = [float(i.replace('%', '')) if i != 'nan%' else 0 for i in all_perf[f'最佳卖出占所有最佳卖点比例']]
            all_perf[f'最佳卖出占所有卖出信号比例'] = [float(i.replace('%', '')) if i != 'nan%' else 0 for i in all_perf[f'最佳卖出占所有卖出信号比例']]
            all_perf[f'最佳买入占所有最佳买点比例'] = [float(i.replace('%', '')) if i != 'nan%' else 0 for i in all_perf[f'最佳买入占所有最佳买点比例']]
            all_perf[f'最佳买入占所有买入信号比例'] = [float(i.replace('%', '')) if i != 'nan%' else 0 for i in all_perf[f'最佳买入占所有买入信号比例']]

            # 精选策略筛选条件1
            # mask1 = (all_perf[f'主动管理超额年化收益率'] > 20) & (all_perf[f'{commission}%费后最大回撤'] < 60)
            # mask2 = (500 > all_perf[f'平均多少天交易一次']) & (all_perf[f'平均多少天交易一次'] > 5)
            # mask3 = (all_perf[f'总胜率'] > 60)
            # mask4 = (all_perf[f'买入信号胜率'] > 55) & (all_perf[f'买入最大亏损'] > -50)
            # mask5 = (all_perf[f'卖出信号胜率'] > 55) & (all_perf[f'卖出最大亏损'] > -50)
            # selected_best_strategy = all_perf[mask1 & mask2 & mask3 & mask4 & mask5]

            # 精选策略筛选条件2
            mask1 = (all_perf[f'主动管理超额年化收益率'] > 30) & (all_perf[f'{commission}%费后最大回撤'] < 60)
            mask2 = (1000 >= all_perf[f'平均多少天交易一次']) & (all_perf[f'平均多少天交易一次'] >= 0)  # holding_pct法下的交易次数设置为0
            # mask2 = (500 > all_perf[f'平均多少天交易一次']) & (all_perf[f'平均多少天交易一次'] > 5)
            mask3 = (all_perf[f'买入最大亏损'] > -50)
            selected_best_strategy = all_perf[mask1 & mask2 & mask3]

            # 买点策略筛选条件1
            mask1 = (all_perf[f'最佳买入占所有最佳买点比例'] >= 75) & (all_perf[f'最佳买入占所有最佳买点比例'] <= 120)
            mask2 = all_perf[f'最佳买入占所有买入信号比例'] >= 50
            mask3 = all_perf[f'回测总天数'] > 1460
            selected_buy_strategy = all_perf[mask1 & mask2 & mask3]

            # 卖点策略筛选条件1
            mask1 = (all_perf[f'最佳卖出占所有最佳卖点比例'] >= 75) & (all_perf[f'最佳卖出占所有最佳卖点比例'] <= 120)
            mask2 = all_perf[f'最佳卖出占所有卖出信号比例'] >= 50
            mask3 = all_perf[f'回测总天数'] > 1460
            selected_sell_strategy = all_perf[mask1 & mask2 & mask3]

            for strategy in [selected_best_strategy, selected_buy_strategy, selected_sell_strategy]:
                if not strategy.empty:
                    strategy[f'{commission}%费后年化收益率'] = [f"{float(i):.2f}%" for i in strategy[f'{commission}%费后年化收益率']]
                    strategy[f'{commission}%费后最大回撤'] = [f"{float(i):.2f}%" for i in strategy[f'{commission}%费后最大回撤']]
                    # strategy[f'主动管理超额年化收益率'] = [f"{float(i):.2f}%" for i in strategy[f'主动管理超额年化收益率']]
                    strategy[f'总胜率'] = [f"{float(i):.2f}%" for i in strategy[f'总胜率']]
                    strategy[f'买入信号胜率'] = [f"{float(i):.2f}%" for i in strategy[f'买入信号胜率']]
                    strategy[f'买入最大亏损'] = [f"{float(i):.2f}%" for i in strategy[f'买入最大亏损']]
                    strategy[f'卖出信号胜率'] = [f"{float(i):.2f}%" for i in strategy[f'卖出信号胜率']]
                    strategy[f'卖出最大亏损'] = [f"{float(i):.2f}%" for i in strategy[f'卖出最大亏损']]
                    # strategy[f'最佳卖出占所有最佳卖点比例'] = [f"{float(i):.2f}%" for i in strategy[f'最佳卖出占所有最佳卖点比例']]
                    # strategy[f'最佳卖出占所有卖出信号比例'] = [f"{float(i):.2f}%" for i in strategy[f'最佳卖出占所有卖出信号比例']]
                    # strategy[f'最佳买入占所有最佳买点比例'] = [f"{float(i):.2f}%" for i in strategy[f'最佳买入占所有最佳买点比例']]
                    # strategy[f'最佳买入占所有买入信号比例'] = [f"{float(i):.2f}%" for i in strategy[f'最佳买入占所有买入信号比例']]

            selected_best_strategy = selected_best_strategy.T
            selected_buy_strategy = selected_buy_strategy.T
            selected_sell_strategy = selected_sell_strategy.T

            if if_print:
                file_name = os.path.join(file_path, f'backtest_result_selected_best_strategy')
                selected_best_strategy.to_csv(f'{file_name}.csv', encoding='utf_8_sig')
                file_name = os.path.join(file_path, f'backtest_result_selected_buy_strategy')
                selected_buy_strategy.to_csv(f'{file_name}.csv', encoding='utf_8_sig')
                file_name = os.path.join(file_path, f'backtest_result_selected_sell_strategy')
                selected_sell_strategy.to_csv(f'{file_name}.csv', encoding='utf_8_sig')
            selected_best_strategy_list.append(selected_best_strategy)
            selected_buy_strategy_list.append(selected_buy_strategy)
            selected_sell_strategy_list.append(selected_sell_strategy)

        all_selected_best_strategy = pd.concat(selected_best_strategy_list, axis=1)
        all_selected_buy_strategy = pd.concat(selected_buy_strategy_list, axis=1)
        all_selected_sell_strategy = pd.concat(selected_sell_strategy_list, axis=1)

        if if_print:
            if factor_name in already_tested_normal_indic:
                file_path = os.path.join(TEST_DIR, f'{factor_name}')
            else:
                file_path = os.path.join(TA_TEST_DIR, f'{factor_name.__name__}')
            os.makedirs(file_path, exist_ok=True)
            file_name = os.path.join(file_path, f'all_selected_best_strategy')
            all_selected_best_strategy.to_csv(f'{file_name}.csv', encoding='utf_8_sig')
            file_name = os.path.join(file_path, f'all_selected_buy_strategy')
            all_selected_buy_strategy.to_csv(f'{file_name}.csv', encoding='utf_8_sig')
            file_name = os.path.join(file_path, f'all_selected_sell_strategy')
            all_selected_sell_strategy.to_csv(f'{file_name}.csv', encoding='utf_8_sig')

    return


def get_all_selected_strategies(indic='all', if_print=False):
    """
    获取所有精选指标对应信号策略， 卖点策略， 买点策略

    :param if_print:
    :return:
    """

    all_selected_best_strategy_list = []
    all_selected_buy_strategy_list = []
    all_selected_sell_strategy_list = []

    already_tested_normal_indic = get_already_tested_indic()
    already_tested_ta_indic = get_already_tested_ta_indic()
    if indic == 'all':
        already_tested_indic = already_tested_normal_indic + already_tested_ta_indic
    elif indic == 'ta':
        already_tested_indic = already_tested_ta_indic
    elif indic == 'normal':
        already_tested_indic = already_tested_normal_indic

    for factor_name in already_tested_indic:
        if factor_name in already_tested_normal_indic:
            factor_name_temp = factor_name
            file_path = os.path.join(TEST_DIR, f'{factor_name}')
        else:
            factor_name_temp = factor_name.__name__
            file_path = os.path.join(TA_TEST_DIR, f'{factor_name.__name__}')
        best_file_name = os.path.join(file_path, f'all_selected_best_strategy')
        buy_file_name = os.path.join(file_path, f'all_selected_buy_strategy')
        sell_file_name = os.path.join(file_path, f'all_selected_sell_strategy')

        if os.path.exists(f'{best_file_name}.csv'):
            selected_best_strategy = pd.read_csv(f'{best_file_name}.csv')  # 数据读取
        else:
            continue
        if os.path.exists(f'{buy_file_name}.csv'):
            selected_buy_strategy = pd.read_csv(f'{buy_file_name}.csv')  # 数据读取
        else:
            continue
        if os.path.exists(f'{sell_file_name}.csv'):
            selected_sell_strategy = pd.read_csv(f'{sell_file_name}.csv')  # 数据读取
        else:
            continue

        for ii, strategy in enumerate([selected_best_strategy, selected_buy_strategy, selected_sell_strategy]):
            selected_strategy_name_list = []
            if strategy.shape[1] > 1:  # 该指标有筛选出来的精选策略, 或者卖点策略，或者卖点策略
                strategy.set_index('Unnamed: 0', inplace=True)
                strategy.index.name = '绩效指标'
                # 剔除full_history加denoise_hp 或者 full_history加detrend_denoise， 因为这些策略使用了未来数据
                for strategy_name in strategy.columns:
                    test1 = ('denoise_hp-full_history' not in strategy_name)  # strategy_name 不是 full_history加denoise_hp
                    test2 = ('detrend_denoise-full_history' not in strategy_name)  # strategy_name 不是 full_history加detrend_denoise
                    test3 = not (('-full_history-' in strategy_name) and (('-2-' in strategy_name) or ('-3-' in strategy_name)))
                            # strategy_name 不是 full_history加2分位， 三分位； 因为全历史滤波过程已经运用了未来数据
                    if test1 and test2 and test3:
                        selected_strategy_name_list.append(strategy_name)

                # 根据回测区间长度剔除一些数据
                if selected_strategy_name_list:
                    selected_strategy = strategy[selected_strategy_name_list]
                    selected_strategy = selected_strategy.T
                    selected_strategy[f'回测总天数'] = selected_strategy[f'回测总天数'].astype(int)
                    mask = (selected_strategy[f'回测总天数'] >= 1460)
                    # mask = (selected_strategy[f'回测总天数'] >= 1095) & (selected_strategy[f'回测总天数'] <= 1460)
                    selected_strategy = selected_strategy[mask]
                else:
                    continue

                # 剔除利用了未来数据的策略，还有筛选出来的策略
                selected_strategy[f'主动管理超额年化收益率'] = selected_strategy[f'主动管理超额年化收益率'].astype(float)
                selected_strategy[f'最佳买入占所有最佳买点比例'] = selected_strategy[f'最佳买入占所有最佳买点比例'].astype(float)
                selected_strategy[f'最佳买入占所有买入信号比例'] = selected_strategy[f'最佳买入占所有买入信号比例'].astype(float)
                selected_strategy[f'最佳卖出占所有最佳卖点比例'] = selected_strategy[f'最佳卖出占所有最佳卖点比例'].astype(float)
                selected_strategy[f'最佳卖出占所有卖出信号比例'] = selected_strategy[f'最佳卖出占所有卖出信号比例'].astype(float)
                selected_strategy = selected_strategy.T

                if not selected_strategy.empty:
                    if ii == 0:  # best_strategy
                        # 如果该指标包含多种信号策略，只取主动管理超额年化收益率最高的那个
                        selected_strategy.sort_values(by='主动管理超额年化收益率', axis=1, ascending=False, inplace=True)
                        num_selected_strategies = len(selected_strategy.columns)
                        selected_strategy = selected_strategy.iloc[:, 0]
                        selected_strategy.name = f'{factor_name_temp}-{selected_strategy.name}'
                        selected_strategy['该指标入选的策略个数'] = num_selected_strategies  # 如果入选策略个数过少，则很有可能是参数过拟合导致的
                        all_selected_best_strategy_list.append(selected_strategy)
                    elif ii == 1:  # 买点策略
                        # 如果该指标包含多种买点信号策略，取准确率最高的那个
                        selected_strategy.sort_values(by=['最佳买入占所有最佳买点比例', '最佳买入占所有买入信号比例'],
                                                      axis=1, ascending=False, inplace=True)
                        num_selected_strategies = len(selected_strategy.columns)
                        selected_strategy = selected_strategy.iloc[:, 0]
                        selected_strategy.name = f'{factor_name_temp}-{selected_strategy.name}'
                        selected_strategy['该指标入选的策略个数'] = num_selected_strategies  # 如果入选策略个数过少，则很有可能是参数过拟合导致的
                        all_selected_buy_strategy_list.append(selected_strategy)
                    elif ii == 2:  # 卖点策略
                        # 如果该指标包含多种卖点信号策略，取准确率最高的那个
                        selected_strategy.sort_values(by=['最佳卖出占所有最佳卖点比例', '最佳卖出占所有卖出信号比例'],
                                                      axis=1, ascending=False, inplace=True)
                        num_selected_strategies = len(selected_strategy.columns)
                        selected_strategy = selected_strategy.iloc[:, 0]
                        selected_strategy.name = f'{factor_name_temp}-{selected_strategy.name}'
                        selected_strategy['该指标入选的策略个数'] = num_selected_strategies  # 如果入选策略个数过少，则很有可能是参数过拟合导致的
                        all_selected_sell_strategy_list.append(selected_strategy)
                else:
                    continue

            else:                               # 该指标没有合适的策略
                continue

    if all_selected_best_strategy_list:
        all_selected_best_strategy = pd.concat(all_selected_best_strategy_list, axis=1)
    else:
        all_selected_best_strategy = pd.DataFrame(index=selected_best_strategy.index)

    if all_selected_buy_strategy_list:
        all_selected_buy_strategy = pd.concat(all_selected_buy_strategy_list, axis=1)
    else:
        all_selected_buy_strategy = pd.DataFrame(index=selected_buy_strategy.index)

    if all_selected_sell_strategy_list:
        all_selected_sell_strategy = pd.concat(all_selected_sell_strategy_list, axis=1)
    else:
        all_selected_sell_strategy = pd.DataFrame(index=selected_sell_strategy.index)

    if if_print:
        if factor_name in already_tested_normal_indic:
            DIR = TEST_DIR
        elif factor_name in already_tested_ta_indic:
            DIR = TA_TEST_DIR
        best_file_name = os.path.join(DIR, f'all_selected_best_strategy')
        buy_file_name = os.path.join(DIR, f'all_selected_buy_strategy')
        sell_file_name = os.path.join(DIR, f'all_selected_sell_strategy')

        all_selected_best_strategy.to_csv(f'{best_file_name}.csv', encoding='utf_8_sig')  # 数据保存
        all_selected_buy_strategy.to_csv(f'{buy_file_name}.csv', encoding='utf_8_sig')  # 数据保存
        all_selected_sell_strategy.to_csv(f'{sell_file_name}.csv', encoding='utf_8_sig')  # 数据保存

    return all_selected_best_strategy, all_selected_buy_strategy, all_selected_sell_strategy


def get_all_selected_strategies_signal_method(selected_strategy=False, if_print=False):
    """
    对筛选出来的指标信号策略方式进行解析成可读取的形式

    :param selected_strategy: all_selected_best_strategy 是否经过了人为再筛选一遍，默认为False
    :param if_print: all_selected_strategy_signal_method 是否保存到本地
    :return:
    """

    if selected_strategy:   # 精选指标信号策略是经过人为筛选一遍的，因此直接读取数据就行
        file_name = os.path.join(TEST_DIR, f'all_selected_best_strategy')
        all_selected_strategies = pd.read_csv(f'{file_name}.csv', index_col='绩效指标')
    else:
        all_selected_strategies = get_all_selected_strategies(indic='all', if_print=if_print)
        all_selected_strategies = all_selected_strategies[0].T
        # 如果入选策略个数过少，则很有可能是参数过拟合导致的
        all_selected_strategies = all_selected_strategies[all_selected_strategies['该指标入选的策略个数'] > 1]
        all_selected_strategies = all_selected_strategies.T

    signal_method_list = []
    for factor_info in all_selected_strategies.columns:
        signal_method_series = pd.Series(dtype='object')
        factor_info1 = factor_info.split('-[')  # 本来直接根据-分割就好，但是因为bins里面可能包含负号，所以先用[]进行分割

        factor_info_left = factor_info1[0].split('-')
        if len(factor_info_left) == 5:
            signal_method_series['factor_name'] = factor_info_left[0]
        else:
            factor_name_length = len(factor_info_left)-4
            factor_name = '-'.join(factor_info_left[0: factor_name_length])
            factor_info_left[0] = factor_name
            for i in range(factor_name_length-1):
                factor_info_left.pop(1)
            signal_method_series['factor_name'] = factor_info_left[0]

        signal_method_series['factor_type'] = factor_info_left[1]
        signal_method_series['preprocess'] = factor_info_left[2][11:]
        signal_method_series['signal_method'] = factor_info_left[3]
        signal_method_series['signal_type'] = factor_info_left[4]

        factor_info_right = factor_info1[1].split(']-')
        if factor_info_right[0] == '':
            signal_method_series['bins'] = []
        else:
            bins = factor_info_right[0].split(',')
            signal_method_series['bins'] = [float(i) for i in bins]
        factor_info_right = factor_info_right[1].split('-')
        signal_method_series['positive'] = True if factor_info_right[0] == 'True' else False
        signal_method_series['left_right'] = factor_info_right[1]  # 左侧交易还是右侧交易

        signal_method_list.append(signal_method_series)

    signal_method_df = pd.concat(signal_method_list, axis=1).T
    signal_method_df.set_index('factor_name', inplace=True)

    if if_print:
        file_name = os.path.join(TEST_DIR, f'all_selected_best_strategy_signal_method')
        signal_method_df.to_csv(f'{file_name}.csv', encoding='utf_8_sig')  # 数据保存

    return signal_method_df


def cluster_strategies(if_plot=False):
    asset = 'BTC'
    start_date = '2015-01-01'
    end_date = get_yesterday(marker='with_n_dash')  # 计算截止昨日收盘

    file_name = os.path.join(TEST_DIR, f'all_selected_best_strategy_signal_method')
    all_timing_signal_method = pd.read_csv(f'{file_name}.csv', index_col='factor_name')
    factor_list = list(all_timing_signal_method.index)

    file_name = os.path.join(TEST_DIR, f'all_selected_best_strategy')
    all_selected_strategy = pd.read_csv(f'{file_name}.csv', index_col='绩效指标')

    self_defined_metrics = onchain_metrics_list + trading_metrics_list + social_metrics_list
    social_metrics = sanbase.get_social_metrics_list(asset='bitcoin')

    factor_df_list = []
    for factor_name in factor_list:
        if factor_name in self_defined_metrics:
            if factor_name in onchain_metrics_list:
                file_path1 = os.path.join(DATA_DIR, f'onchain_data')
            elif factor_name in trading_metrics_list:
                file_path1 = os.path.join(DATA_DIR, f'trading_data')
            elif factor_name in social_metrics_list:
                file_path1 = os.path.join(DATA_DIR, f'social_data')
            else:
                raise NotImplementedError
            file_name1 = os.path.join(file_path1, f'{factor_name}')
            origin_factor_df = pd.read_csv(f'{file_name1}.csv', index_col='end_date')
            origin_factor_df = origin_factor_df.loc[start_date:end_date, :]
        elif factor_name in social_metrics:
            # origin_factor_df = sanbase.get_indicators(indic_name=factor_name, asset=asset, start_date=start_date, end_date=end_date)
            file_path1 = os.path.join(DATA_DIR, f'social_data')
            file_name1 = os.path.join(file_path1, f'{factor_name}')
            origin_factor_df = pd.read_csv(f'{file_name1}.csv', index_col='end_date')
            origin_factor_df = origin_factor_df.loc[start_date:end_date, :]
        else:
            origin_factor_df = get_indicators(indic_name=factor_name, asset=asset, start_date=start_date, end_date=end_date)
        factor_df_list.append(origin_factor_df)
    factor_df = pd.concat(factor_df_list, axis=1)
    file_name = os.path.join(DATA_DIR, f'精选指标数据')
    factor_df.to_csv(f'{file_name}.csv', encoding='utf_8_sig')

    # file_name = os.path.join(DATA_DIR, f'精选指标数据')
    # factor_df = pd.read_csv(f'{file_name}.csv', index_col=0)
    cluster_df = factor_clustering(factor_df)

    selected_factor_list = []
    selected_strategy_list = []
    for cluster_num in cluster_df['label'].unique():
        cluster = cluster_df[cluster_df['label'] == cluster_num]
        this_cluster_factor_df = factor_df[cluster.index]

        if if_plot:
            sns.heatmap(this_cluster_factor_df.corr(), annot=True)
            plt.show()
            plt.close()

        this_cluster_strategies = []
        for strategy in all_selected_strategy.columns:
            test = sum([i in strategy for i in cluster.index])
            if test > 0:
                this_cluster_strategies.append(strategy)
        this_cluster_all_selected_strategy = all_selected_strategy[this_cluster_strategies]
        this_cluster_all_selected_strategy = this_cluster_all_selected_strategy.T
        this_cluster_all_selected_strategy[f'总胜率'] = [float(i.replace('%', '')) for i in this_cluster_all_selected_strategy[f'总胜率']]
        this_cluster_all_selected_strategy.sort_values(by=['总胜率', '主动管理超额年化收益率'], axis=0, ascending=False, inplace=True)
        this_cluster_all_selected_strategy = this_cluster_all_selected_strategy.T
        this_cluster_elected_strategy = this_cluster_all_selected_strategy.iloc[:, 0]
        selected_strategy_list.append(this_cluster_elected_strategy)
        selected_factor = cluster.index[[i in this_cluster_elected_strategy.name for i in cluster.index]][0]
        selected_factor_list.append(selected_factor)

    all_selected_strategy_after_cluster = pd.concat(selected_strategy_list, axis=1)
    file_name = os.path.join(TEST_DIR, f'all_selected_strategy_after_cluster')
    all_selected_strategy_after_cluster.to_csv(f'{file_name}.csv', encoding='utf_8_sig')
    all_selected_strategy_signal_method_after_cluster = all_timing_signal_method.loc[selected_factor_list, :]
    file_name = os.path.join(TEST_DIR, f'all_selected_strategy_signal_method_after_cluster')
    all_selected_strategy_signal_method_after_cluster.to_csv(f'{file_name}.csv', encoding='utf_8_sig')
    file_name = os.path.join(TEST_DIR, f'all_selected_strategy_cluster')
    cluster_df.to_csv(f'{file_name}.csv', encoding='utf_8_sig')

    return cluster_df


if __name__ == '__main__':
    # 测试delete_each_indic_selected_strategies
    # delete_each_indic_selected_strategies()

    # 测试get_each_indic_selected_strategies, 获取每个指标的selected_strategies
    all_selected_strategies = get_each_indic_selected_strategies(indic='ta', if_print=True)

    # 测试get_all_selected_strategies， 获取全部指标的selected_strategies（基于一些筛选条件）
    all_selected_strategies = get_all_selected_strategies(indic='ta', if_print=True)

    # get_all_selected_strategies_signal_method, 全部指标的selected_strategies的信号转换方法解析出来
    # all_signal_method = get_all_selected_strategies_signal_method(selected_strategy=True, if_print=True)

    # 测试cluster_strategies, 对全部指标的selected_strategies进行算法聚类，不过还是建议手动聚类
    # cluster_strategies(if_plot=True)